package main

import (
	"extractor/core/downloader"
	"extractor/core/engine"
	"extractor/db"
	"extractor/zuidzy"
	"fmt"
	"time"
)

func main() {

	// 连接数据库
	//err := db.ConnectDB("root", "password", "database", "localhost:3306")
	//if err != nil {
	//	log.Println("connect database fail")
	//	return
	//}

	p := &zuidzy.Zuidazy{}
	urls := make([]engine.UrlWithParams, 0)
	for i := 3; i < 5; i++ {
		urls = append(urls, engine.UrlWithParams{
			Url:    p.GetListUrl(i, 1),
			Params: i,
		})
	}

	fmt.Println(time.Now().Unix())

	engine.NewEngine(&zuidzy.Zuidazy{}).
		PutUrls(urls, downloader.TEXT).
		SetPipeline(zuidzy.NewFilePipeline()).
		SetThreadnum(11).
		Run()

	fmt.Println(time.Now().Unix())

	// 去除重复的数据
	db.RemoveDup()

	fmt.Println(time.Now().Unix())

}
